#!/bin/bash


# 
# This will scrape a range of pages to build a basic index of TED talks.  
# Each line is output as a self-contained PHP block to prevent errors.  

# Example call: 

# curl 'http://www.ted.com/talks/{podtv,view}/id/[188-191]' --create-dirs -o "ted_#2_#1.txt"

echo -e "\n\nrunning tedtanks.sh at `date`\n";

# LOAD ARGUMENTS INTO VARIABLES

# s = start ID
# e = end ID
# o = output file
while getopts "s:e:o:" OPTS
do
  case "$OPTS" in
    s) START="$OPTARG";;
    e) END="$OPTARG";;
    o) OUTFILE="$OPTARG";;
  esac
done

# fail if no outfile specified
if [[ -z $OUTFILE ]]
    then
    echo "Error: An output file must be specifed" 1>&2 
    exit 1;
fi


# assign defaults if flags aren't set
if [[ -z $START ]]; then START=1; fi;
if [[ -z $END ]]; then END=$START; fi;    # default to getting one ID
    
# reverse values if END is before START
if [[ $START -ge $END ]]
    then
    TEMP=$START;
    START=$END;
    END=$TEMP;
fi

# echo 'Start: '$START' End: '$END;
# exit 1


# initialize counter for 404 errors
FAILCOUNT=0;

for ((ID=$START; ID <= $END ; ID++))
do
    
    # check to see if the ID is already listed in the outfile, if it is, skip everything else
    if grep -q '$TED\['$ID'\]' $OUTFILE

        then
        echo "Skipping $ID, entry exists"
        FAILCOUNT=0;
    else
        # sleep a few seconds to keep from DOSing the TED servers
        sleep 5;
    
        # increment the 404 counter
        let "FAILCOUNT++";
    
        # report where we're at
        echo "Fetching $ID"
        
        # curl to grab the MP4 url and duration from the iTunes podcast:
        CURL=`curl -s http://www.ted.com/talks/podtv/id/${ID} | sed -r -n -e'/<enclosure/{s/^.*(http:\/\/ted.streamguys.net[^<]+_[0-9]{4}[A-Z]?\.mp4).*$/MP4="\1"\;/;h;};/<itunes:duration>/{s/^.*>([0-9:]{8})<.*$/DUR="\1"/;H;x;s/\n/ /gp;}'`;

        # set variables from values returned in CURL
        if [[ "${CURL:0:3}" == 'MP4' ]]
            then
            eval $CURL;
        fi
    
        # if we don't find an MP4 file, there's no point in going on...
        if [[ -z $MP4 ]]
            then
            echo "http://www.ted.com/talks/podtv/id/${ID}: 404  (failcount $FAILCOUNT)" 1>&2;
            else


            # curl to grab the FLV url, the flv filename, thumbnail image and the title 
            # eval the result to set vars
            CURL=`curl -s http://www.ted.com/talks/view/id/${ID} | sed -r -n -e'/<title>/{s/.*TED \| Talks \| (.*) \(video\).*/TITLE="\1"\;/;H;};   /[0-9]+_389x292.jpg/{s/^.*(http:\/\/media.ted.com\/images\/ted\/[0-9]+_389x292.jpg).*$/JPG="\1"\; /;H;};   /static.videoegg.com.*flv/{s/^.*&file=(http:\/\/static.videoegg.com\/ted\/movies\/([^.]+\.flv))&.*$/TED="\2"\; FLV="\1"/;H;};/flashstream.adobe.com.*flv/{s/^.*tedtalks%5C%2F([^.]+[0-9]+[A_Z]?\.flv)%22.*$/ALT="\1"; /;H;};   ${x;s/\n/ /gp;}'`;

            # set variables from values returned in CURL
            eval $CURL;

            #    That's compressed from these commands:
            #   
            #   /<title>/{
            #   s/.*TED \| Talks \| (.*) \(video\).*/TITLE="\1"\;/
            #   H
            #   }
            #   
            #   /[0-9]+_389x292.jpg/{
            #   s/^.*(http:\/\/media.ted.com\/images\/ted\/[0-9]+_389x292.jpg).*$/JPG="\1"\; /
            #   H
            #   }
            #   
            #   /static.videoegg.com.*flv/{
            #   s/^.*&file=(http:\/\/static.videoegg.com\/ted\/movies\/([^.]+\.flv))&.*$/TED="\2"\; FLV="\1"/
            #   H
            #   }
            #   
            #   # get ALT
            #   /flashstream.adobe.com.*flv/{
            #   s/^.*tedtalks%5C%2F([^.]+[0-9]+[A_Z]?\.flv)%22.*$/ALT="\1"; /
            #   H
            #   }
            #   
            #   $ {
            #   	x
            #   	s/\n/ /gp
            #   }
            #
            #  result is returned as shell variable assignments, so eval the result
                
            # make sure there are no empty keys
            if [[ -z $TED ]]; then TED=$ALT; fi;
        
            echo '<?php $TED['$ID'] = $TED['"'${TED}'"'] = array("id" => '$ID', "flv" => "'$FLV'", "title" => "'$TITLE'", "jpg" => "'$JPG'", "mp4" => "'$MP4'", "duration" => "'$DUR'", "page" => "http://www.ted.com/talks/view/id/'$ID'"); ?>' >> $OUTFILE
        
            # reset the 404 counter
            FAILCOUNT=0;
        fi
    
        # check 404 counter, error out if greater than 6 ( needs to accomodate the gap between 132 and 138 )
        if [[ $FAILCOUNT -gt 6 ]]
            then
            echo "Error: More than 6 consecutive 404 errors" 1>&2 
            exit 1;
        fi
    
        # cleanup for the next loop
        unset FLV TITLE JPG MP4 DUR ALT TED CURL;
    # end the grep for id conditional
    fi
done
